#----------------------------#
#!Python3
# coding:utf-8
# target: grap yhd pictures
# author: terry
#----------------------------#

import requests
from lxml.html import fromstring
import json
import csv
from requests import HTTPError,ConnectionError
import time

#region --analysising:
# # 类别xpath内容  /html/body/div[1]/div[2]/div/div[2]/div[3]/div/div[2]/div[1]/div[1]/div/ul/li[1]/a
# 分类信息ajax异步加载

# 分类目标的展示,网页展示了部分内容
# target_1 乳品生鲜
# https://search.yhd.com/c0-0-1003298/
# https://search.yhd.com/c0-0-1003298/#page=2&sort=1
# https://search.yhd.com/c0-0-1003298/#page=3&sort=1
# 2 酒水饮料，
# p1=https://search.yhd.com/c0-0-1004071/
# p2=https://search.yhd.com/c0-0-1004071/#page=2&sort=1
# p3=https://search.yhd.com/c0-0-1004071/#page=3&sort=1
# 3 饼干糕点，
# https://search.yhd.com/c0-0-1004174/
# 4 休闲零食
# https://search.yhd.com/c0-0-1004099/

#图片地址：
#商品柜 >https://img14.360buyimg.com/n7/s230x230_jfs/t5608/147/7235798863/128981/e47a121e/59701d64N26fa167e.jpg!cc_230x230.jpg
#商品介绍页：
#默认显示的地址（仅有货）
#全部显示的地址

# https://search.yhd.com/c0-0-1004174/mbname-b/a-s1-v4-p1-price-d0-f0b-m1-rt0-pid-mid0-color-size-k/#page=50&sort=1

# https://search.yhd.com/c0-0-1004099/mbname-b/a-s1-v4-p1-price-d0-f0-m1-rt0-pid-mid0-color-size-k/
# https://search.yhd.com/c0-0-1004071/mbname-b/a-s1-v4-p1-price-d0-f0-m1-rt0-pid-mid0-color-size-k/#page=49&sort=1
# https://search.yhd.com/c0-0-1003298/mbname-b/a-s1-v4-p1-price-d0-f0-m1-rt0-pid-mid0-color-size-k/

#endregion


# compose the address
domain="yhd.com"
base_page="http://search.yhd.com/"
c=["c0-0-1003847","c0-0-1003327","c0-0-1003746","c0-0-1004802"]
suffix='/mbname-b/a-s1-v4-p1-price-d0-f0-m1-rt0-pid-mid0-color-size-k/'
page_limit=50

urls=[]
for i in range(page_limit):
    url=base_page+c[0]+suffix+"#page="+str(i+1)+"&sort=1"
    # print(url)
    urls.append(url)


# compose the request

ua=[
    {"User-agent":"(Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18362" },
    {"User-agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3741.400 QQBrowser/10.5.3863.400"},
    {"User-agent":"Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/69.0"}
  ]

target_page=urls[7]
try:
    res=requests.get(target_page,headers=ua[1])
    data=fromstring(res.text)
    #print(res.text)
    #抓取数据 div[@id="itemSearchList"]
    selector=data.xpath('//div[@class="mod_search_pro" and ]')
    #('//div[@class="mod_search_pro"]/div[@class="itemBox"]/p[1]/em/@yhdprice|//div[@class="mod_search_pro",@defaultid="0"]/div[@class="itemBox"]/p[1]/em/@yhdprice')
    print( "num of s1:",len(selector))

    s2=data.xpath('//div[@class="mod_search_pro" and @defaultid="0"]')
    print("num of s2:",len(s2))
    # for i in selector:
    #     print("price tag:",i)
        
    #name_selector=data.xpath('//div[@class="mod_search_pro"]/div/p[@class="proName clearfix"]/a//@title')
    # price_selector=data.xpath('//div[@id="itemSearchList"]/div[@class=mod_search_pro]/div/p[@class="proPrice"]/em/text()')
    #('//div[@class="mod_search_pro"]/div[@class="itemBox"/p[@class="proPrice"]/em/text()')
    # thumb_link_selector=/div[@class="mod_search_pro"]/div/div[@id="searchProImg"]/a/img@href
    # detail_link_selector=/div[@class="mod_search_pro"]/div/div[@id="searchProImg"]/a@
    # print( len(price_selector))
    # print(res.url)
    # for item in price_selector:
    #     print (item)
    
except HTTPError as e:
    print(e)
print( "!!! Job Done!")

""" with open('yhd_records.cvs','w+') as f:
    f.writelines("Title","Price","image_link","product_link")
    try:
        for item in results:
        f.writelines(item)
    except expression as e:
        print(e) """